library(tidyverse)
── Attaching core tidyverse packages ──────────────────────────────────────────────────────────────────────────────────────────────────────── tidyverse 2.0.0 ──
✔ dplyr 1.1.4 ✔ readr 2.1.5
✔ forcats 1.0.0 ✔ stringr 1.5.1
✔ ggplot2 3.5.2 ✔ tibble 3.3.0
✔ lubridate 1.9.4 ✔ tidyr 1.3.1
✔ purrr 1.1.0
── Conflicts ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag() masks stats::lag()
ℹ Use the ]8;;http://conflicted.r-lib.org/conflicted package]8;; to force all conflicts to become errors
library(car)
G3;Loading required package: carData
gG3;
Attaching package: ‘car’
gG3;The following object is masked from ‘package:dplyr’:
recode
gG3;The following object is masked from ‘package:purrr’:
some
g
library(performance)
library(patchwork)
idea:
- sim data that violates normality of errors and homosced.
- have students identify those errors based on diagnostic plots
- bootstrap data from this dataset
- plot it and see if those assumptions change
Sim violating data
Something different with the errors this time. Give students new
illustration of how things can diverge from normality. Not asymmetrical,
but maybe U-shaped errors? As in, mostly large in both directions, not
usually small?
shapes <- .3
set.seed(1)
beta_errors <- rbeta(101, shape1 = shapes, shape2 = shapes) |>
datawizard::standardize()
# scale(scale = FALSE, center = TRUE)
plot(beta_errors)

hist(beta_errors)

set.seed(1)
error_inc_terms <- seq(.5, 3, length.out = 101)
df <- tibble(
x = seq(-2, 2, length.out = 101),
y_good = 2 + (-1.1 * x) + rnorm(101, 0, 1),
y_viol = 2 + (-1.1 * x) + beta_errors * error_inc_terms,
)
p_good <- df |>
ggplot(aes(x=x, y=y_good)) +
geom_point() +
# geom_smooth(method = 'lm', se = FALSE) +
NULL
p_beta <- df |>
ggplot(aes(x=x, y=y_viol)) +
geom_point() +
# geom_smooth(method = 'lm', se = FALSE) +
NULL
p_good + p_beta

Fit models
m_good <- lm(y_good ~ x, data = df)
m_viol <- lm(y_viol ~ x, data = df)
summary(m_viol)
Call:
lm(formula = y_viol ~ x, data = df)
Residuals:
Min 1Q Median 3Q Max
-4.0231 -1.3473 0.3146 1.5062 3.0814
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) 2.1154 0.1846 11.459 < 2e-16 ***
x -0.8130 0.1583 -5.136 1.41e-06 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 1.855 on 99 degrees of freedom
Multiple R-squared: 0.2104, Adjusted R-squared: 0.2024
F-statistic: 26.38 on 1 and 99 DF, p-value: 1.408e-06
Plot diagnostics
check_model(m_viol)

Q-Q plot
plot(m_viol, which = 2)

Histogram of residuals
- asymmetrical
- fatter tails than we would expect
hist(m_viol$residuals)

It’s really interesting how the model can find a line such that the
residuals look almost gaussian, even though the generative process was
so far from gaussian.
Compare residuals to fitted values
plot(m_viol, which = 1)

car::Boot()
fit model
In reality, present this after showing the manual process.
Boot() draws R samples with the same number
of observations as the original dataset.
set.seed(1)
m_viol_boot <- Boot(m_viol, R = 1000)
G3;Loading required namespace: boot
g
summary(m_viol_boot)
Number of bootstrap replications R = 1000
original bootBias bootSE bootMed
(Intercept) 2.11540 0.00524979 0.18235 2.13003
x -0.81301 -0.00071738 0.16164 -0.81038
The SE of the bootstrapped version is smaller – there is less
variability in the bootstrapped data than in the original sample. Here’s
the SE of the original sample:
summary(m_viol)$coefficients
Estimate Std. Error t value Pr(>|t|)
(Intercept) 2.1153955 0.1846088 11.458803 7.444834e-20
x -0.8130144 0.1583007 -5.135886 1.408443e-06
Confint(m_viol_boot, level = 0.95, type = 'perc')
Bootstrap percent confidence intervals
Consequently, the CIs are narrower for the bootstrapped model than
they are for the original model. Original CIs:
Confint(m_viol, level = 0.95, type = 'perc')
Estimate 2.5 % 97.5 %
(Intercept) 2.1153955 1.749092 2.4816994
x -0.8130144 -1.127117 -0.4989114
plot diagnostics?
Doesn’t make a ton of sense because doesn’t really rely on same
assumptions as the parametric version of the model.
plot(m_viol_boot)

Interesting, so apparently we want the t-values to be normally
distributed.
check_model() throws an error.
# check_model(m_viol_boot)
fail: emmeans doesn’t work on boot object.
# m_viol_emm <- emmeans( ref_grid(m_viol, at = list(x = c(-2, -1, 0, 1, 2))), ~x)
# m_viol_emm |> as.data.frame()
# m_viol_boot_emm <- emmeans( ref_grid(m_viol_boot, at = list(x = c(-2, -1, 0, 1, 2))), ~x)
Manually bootstrap from this data
- bootstrap sample
- fit model to that data
- gather values of important coefs
- make distrib of those coefs
- summarise those distribs
- that’s what the model is doing
boot_sample_size <- 25
n_resamples <- 50
orig_data <- df |>
select(x, y_viol) |>
rowid_to_column()
draw_boot_samples <- function(dataset, resample_size, n_resamples){
# dataset: dataframe or tibble
# resample_size: integer, size of the sample to draw
# n_resamples: integer, number of bootstrap samples to draw
# returns list with each element a subset of dataset
boot_accum <- list()
for(i in 1:n_resamples){
boot_accum[[i]] <- dataset |>
slice_sample(n = resample_size, replace = TRUE)
}
return(boot_accum)
}
plot_boot_sample <- function(boot_sample){
# boot_sample: dataset, outcome of draw_boot_sample
boot_sample |>
ggplot(aes(x = x, y = y_viol)) +
geom_point() +
geom_smooth(method = 'lm', se = FALSE) +
xlim(-2, 2) +
NULL
}
set.seed(1)
boot_samples <- draw_boot_samples(orig_data, boot_sample_size, n_resamples)
boot_plot_list <- lapply(boot_samples, plot_boot_sample)
# plot_boot_sample(boot_samples[[1]])
wrap_plots(boot_plot_list[1:12])

fit_lm <- function(boot_sample){
# boot_sample: df with cols x, y_viol
# returns coefs of LM fit to boot_sample
m <- lm(y_viol ~ x, data = boot_sample)
return(m$coefficients)
}
# fit_lm(boot_samples[[1]])
boot_coefs <- lapply(boot_samples, fit_lm)
boot_coefs_df <- boot_coefs |>
bind_rows(.id = 'boot_idx')
boot_int_mean <- mean(boot_coefs_df$`(Intercept)`)
boot_int_sd <- sd(boot_coefs_df$`(Intercept)`)
boot_slp_mean <- mean(boot_coefs_df$x)
boot_slp_sd <- sd(boot_coefs_df$x)
p_intercept <- boot_coefs_df |>
ggplot(aes(x = `(Intercept)`)) +
geom_histogram(fill = 'grey') +
geom_vline(xintercept = boot_int_mean, linewidth = 2, colour = 'black') +
geom_vline(xintercept = boot_int_mean + boot_int_sd, linewidth = 1, colour = 'black', linetype = 'dashed') +
geom_vline(xintercept = boot_int_mean - boot_int_sd, linewidth = 1, colour = 'black', linetype = 'dashed') +
geom_vline(xintercept = boot_int_mean + 1.96*boot_int_sd, linewidth = 1, colour = 'black', linetype = 'dotted') +
geom_vline(xintercept = boot_int_mean - 1.96*boot_int_sd, linewidth = 1, colour = 'black', linetype = 'dotted') +
geom_function(fun = function(x) dnorm(x, mean = boot_int_mean, sd = boot_int_sd) * 3, colour = 'black') +
NULL
p_slope <- boot_coefs_df |>
ggplot(aes(x = x)) +
geom_histogram(fill = 'grey') +
geom_vline(xintercept = boot_slp_mean, linewidth = 2, colour = 'black') +
geom_vline(xintercept = boot_slp_mean + boot_slp_sd, linewidth = 1, colour = 'black', linetype = 'dashed') +
geom_vline(xintercept = boot_slp_mean - boot_slp_sd, linewidth = 1, colour = 'black', linetype = 'dashed') +
geom_vline(xintercept = boot_slp_mean + 1.96*boot_slp_sd, linewidth = 1, colour = 'black', linetype = 'dotted') +
geom_vline(xintercept = boot_slp_mean - 1.96*boot_slp_sd, linewidth = 1, colour = 'black', linetype = 'dotted') +
geom_function(fun = function(x) dnorm(x, mean = boot_slp_mean, sd = boot_slp_sd) * 3, colour = 'black') +
NULL
p_intercept + p_slope

^ would be cool to animate the way these sampling distributions
change as samples trickle in, the way I did for bayes_stat.
The big difficulty with bootstrapping is that to actually understand
WHY it works, you need to understand the very very complicated maths
behind it. So it’s not easy to give a good intuition for why it
helps.
LS0tCnRpdGxlOiAiTGVjdHVyZSAwOSBwbGF5Z3JvdW5kIgpvdXRwdXQ6IAogIGh0bWxfbm90ZWJvb2s6CiAgICB0b2M6IHRydWUKLS0tCgpgYGB7cn0KbGlicmFyeSh0aWR5dmVyc2UpCmxpYnJhcnkoY2FyKQpsaWJyYXJ5KHBlcmZvcm1hbmNlKQpsaWJyYXJ5KHBhdGNod29yaykKYGBgCgppZGVhOgoKLSBzaW0gZGF0YSB0aGF0IHZpb2xhdGVzIG5vcm1hbGl0eSBvZiBlcnJvcnMgYW5kIGhvbW9zY2VkLgotIGhhdmUgc3R1ZGVudHMgaWRlbnRpZnkgdGhvc2UgZXJyb3JzIGJhc2VkIG9uIGRpYWdub3N0aWMgcGxvdHMKLSBib290c3RyYXAgZGF0YSBmcm9tIHRoaXMgZGF0YXNldAotIHBsb3QgaXQgYW5kIHNlZSBpZiB0aG9zZSBhc3N1bXB0aW9ucyBjaGFuZ2UKCgojIFNpbSB2aW9sYXRpbmcgZGF0YQoKU29tZXRoaW5nIGRpZmZlcmVudCB3aXRoIHRoZSBlcnJvcnMgdGhpcyB0aW1lLgpHaXZlIHN0dWRlbnRzIG5ldyBpbGx1c3RyYXRpb24gb2YgaG93IHRoaW5ncyBjYW4gZGl2ZXJnZSBmcm9tIG5vcm1hbGl0eS4KTm90IGFzeW1tZXRyaWNhbCwgYnV0IG1heWJlIFUtc2hhcGVkIGVycm9ycz8gQXMgaW4sIG1vc3RseSBsYXJnZSBpbiBib3RoIGRpcmVjdGlvbnMsIG5vdCB1c3VhbGx5IHNtYWxsPwoKYGBge3J9CnNoYXBlcyA8LSAuMwpzZXQuc2VlZCgxKQpiZXRhX2Vycm9ycyA8LSByYmV0YSgxMDEsIHNoYXBlMSA9IHNoYXBlcywgc2hhcGUyID0gc2hhcGVzKSB8PgogIGRhdGF3aXphcmQ6OnN0YW5kYXJkaXplKCkKICAjIHNjYWxlKHNjYWxlID0gRkFMU0UsIGNlbnRlciA9IFRSVUUpCnBsb3QoYmV0YV9lcnJvcnMpCmhpc3QoYmV0YV9lcnJvcnMpCmBgYAoKYGBge3IgbWVzc2FnZSA9IEZ9CnNldC5zZWVkKDEpCgplcnJvcl9pbmNfdGVybXMgPC0gc2VxKC41LCAzLCBsZW5ndGgub3V0ID0gMTAxKQoKZGYgPC0gdGliYmxlKAogIHggPSBzZXEoLTIsIDIsIGxlbmd0aC5vdXQgPSAxMDEpLAogIHlfZ29vZCA9IDIgKyAoLTEuMSAqIHgpICsgcm5vcm0oMTAxLCAwLCAxKSwgCiAgeV92aW9sID0gMiArICgtMS4xICogeCkgKyBiZXRhX2Vycm9ycyAqIGVycm9yX2luY190ZXJtcywKKQoKcF9nb29kIDwtIGRmIHw+CiAgZ2dwbG90KGFlcyh4PXgsIHk9eV9nb29kKSkgKwogIGdlb21fcG9pbnQoKSArCiAgIyBnZW9tX3Ntb290aChtZXRob2QgPSAnbG0nLCBzZSA9IEZBTFNFKSArCiAgTlVMTAoKCnBfYmV0YSA8LSBkZiB8PgogIGdncGxvdChhZXMoeD14LCB5PXlfdmlvbCkpICsKICBnZW9tX3BvaW50KCkgKwogICMgZ2VvbV9zbW9vdGgobWV0aG9kID0gJ2xtJywgc2UgPSBGQUxTRSkgKwogIE5VTEwKCnBfZ29vZCArIHBfYmV0YQpgYGAKCiMjIEZpdCBtb2RlbHMKCmBgYHtyfQptX2dvb2QgPC0gbG0oeV9nb29kIH4geCwgZGF0YSA9IGRmKQptX3Zpb2wgPC0gbG0oeV92aW9sIH4geCwgZGF0YSA9IGRmKQpgYGAKCmBgYHtyfQpzdW1tYXJ5KG1fdmlvbCkKYGBgCgoKCiMjIFBsb3QgZGlhZ25vc3RpY3MKCmBgYHtyIGZpZy5oZWlnaHQgPSAxMH0KY2hlY2tfbW9kZWwobV92aW9sKQpgYGAKCgoKIyMjIFEtUSBwbG90CgpgYGB7cn0KcGxvdChtX3Zpb2wsIHdoaWNoID0gMikKYGBgCgoKIyMjIEhpc3RvZ3JhbSBvZiByZXNpZHVhbHMKCi0gYXN5bW1ldHJpY2FsCi0gZmF0dGVyIHRhaWxzIHRoYW4gd2Ugd291bGQgZXhwZWN0CgpgYGB7cn0KaGlzdChtX3Zpb2wkcmVzaWR1YWxzKQpgYGAKCkl0J3MgcmVhbGx5IGludGVyZXN0aW5nIGhvdyB0aGUgbW9kZWwgY2FuIGZpbmQgYSBsaW5lIHN1Y2ggdGhhdCB0aGUgcmVzaWR1YWxzIGxvb2sgYWxtb3N0IGdhdXNzaWFuLCBldmVuIHRob3VnaCB0aGUgZ2VuZXJhdGl2ZSBwcm9jZXNzIHdhcyBzbyBmYXIgZnJvbSBnYXVzc2lhbi4KCgojIyMgQ29tcGFyZSByZXNpZHVhbHMgdG8gZml0dGVkIHZhbHVlcwoKYGBge3J9CnBsb3QobV92aW9sLCB3aGljaCA9IDEpCmBgYAoKCiMgY2FyOjpCb290KCkKCiMjIGZpdCBtb2RlbAoKSW4gcmVhbGl0eSwgcHJlc2VudCB0aGlzIGFmdGVyIHNob3dpbmcgdGhlIG1hbnVhbCBwcm9jZXNzLgoKYEJvb3QoKWAgZHJhd3MgYFJgIHNhbXBsZXMgd2l0aCB0aGUgc2FtZSBudW1iZXIgb2Ygb2JzZXJ2YXRpb25zIGFzIHRoZSBvcmlnaW5hbCBkYXRhc2V0LgoKYGBge3J9CnNldC5zZWVkKDEpCm1fdmlvbF9ib290IDwtIEJvb3QobV92aW9sLCBSID0gMTAwMCkKc3VtbWFyeShtX3Zpb2xfYm9vdCkKYGBgClRoZSBTRSBvZiB0aGUgYm9vdHN0cmFwcGVkIHZlcnNpb24gaXMgc21hbGxlciAtLSB0aGVyZSBpcyBsZXNzIHZhcmlhYmlsaXR5IGluIHRoZSBib290c3RyYXBwZWQgZGF0YSB0aGFuIGluIHRoZSBvcmlnaW5hbCBzYW1wbGUuCkhlcmUncyB0aGUgU0Ugb2YgdGhlIG9yaWdpbmFsIHNhbXBsZToKCmBgYHtyfQpzdW1tYXJ5KG1fdmlvbCkkY29lZmZpY2llbnRzCmBgYAoKCmBgYHtyfQpDb25maW50KG1fdmlvbF9ib290LCBsZXZlbCA9IDAuOTUsIHR5cGUgPSAncGVyYycpCmBgYAoKQ29uc2VxdWVudGx5LCB0aGUgQ0lzIGFyZSBuYXJyb3dlciBmb3IgdGhlIGJvb3RzdHJhcHBlZCBtb2RlbCB0aGFuIHRoZXkgYXJlIGZvciB0aGUgb3JpZ2luYWwgbW9kZWwuCk9yaWdpbmFsIENJczoKCmBgYHtyfQpDb25maW50KG1fdmlvbCwgbGV2ZWwgPSAwLjk1LCB0eXBlID0gJ3BlcmMnKQpgYGAKCgojIyBwbG90IGRpYWdub3N0aWNzPwoKRG9lc24ndCBtYWtlIGEgdG9uIG9mIHNlbnNlIGJlY2F1c2UgZG9lc24ndCByZWFsbHkgcmVseSBvbiBzYW1lIGFzc3VtcHRpb25zIGFzIHRoZSBwYXJhbWV0cmljIHZlcnNpb24gb2YgdGhlIG1vZGVsLgoKYGBge3J9CnBsb3QobV92aW9sX2Jvb3QpCmBgYAoKSW50ZXJlc3RpbmcsIHNvIGFwcGFyZW50bHkgd2Ugd2FudCB0aGUgdC12YWx1ZXMgdG8gYmUgbm9ybWFsbHkgZGlzdHJpYnV0ZWQuCgpgY2hlY2tfbW9kZWwoKWAgdGhyb3dzIGFuIGVycm9yLgoKYGBge3J9CiMgY2hlY2tfbW9kZWwobV92aW9sX2Jvb3QpCmBgYAoKCgojIyBmYWlsOiBlbW1lYW5zIGRvZXNuJ3Qgd29yayBvbiBib290IG9iamVjdC4KCmBgYHtyfQojIG1fdmlvbF9lbW0gPC0gZW1tZWFucyggcmVmX2dyaWQobV92aW9sLCBhdCA9IGxpc3QoeCA9IGMoLTIsIC0xLCAwLCAxLCAyKSkpLCB+eCkKIyBtX3Zpb2xfZW1tIHw+IGFzLmRhdGEuZnJhbWUoKQpgYGAKCmBgYHtyfQojIG1fdmlvbF9ib290X2VtbSA8LSBlbW1lYW5zKCByZWZfZ3JpZChtX3Zpb2xfYm9vdCwgYXQgPSBsaXN0KHggPSBjKC0yLCAtMSwgMCwgMSwgMikpKSwgfngpCmBgYAoKCgoKCiMgTWFudWFsbHkgYm9vdHN0cmFwIGZyb20gdGhpcyBkYXRhCgotIGJvb3RzdHJhcCBzYW1wbGUKLSBmaXQgbW9kZWwgdG8gdGhhdCBkYXRhCi0gZ2F0aGVyIHZhbHVlcyBvZiBpbXBvcnRhbnQgY29lZnMgCi0gbWFrZSBkaXN0cmliIG9mIHRob3NlIGNvZWZzIAotIHN1bW1hcmlzZSB0aG9zZSBkaXN0cmlicwotIHRoYXQncyB3aGF0IHRoZSBtb2RlbCBpcyBkb2luZwoKYGBge3IgbWVzc2FnZSA9IEZ9CmJvb3Rfc2FtcGxlX3NpemUgPC0gMjUKbl9yZXNhbXBsZXMgPC0gNTAKCm9yaWdfZGF0YSA8LSBkZiB8PgogIHNlbGVjdCh4LCB5X3Zpb2wpIHw+CiAgcm93aWRfdG9fY29sdW1uKCkKCmRyYXdfYm9vdF9zYW1wbGVzIDwtIGZ1bmN0aW9uKGRhdGFzZXQsIHJlc2FtcGxlX3NpemUsIG5fcmVzYW1wbGVzKXsKICAjIGRhdGFzZXQ6IGRhdGFmcmFtZSBvciB0aWJibGUKICAjIHJlc2FtcGxlX3NpemU6IGludGVnZXIsIHNpemUgb2YgdGhlIHNhbXBsZSB0byBkcmF3CiAgIyBuX3Jlc2FtcGxlczogaW50ZWdlciwgbnVtYmVyIG9mIGJvb3RzdHJhcCBzYW1wbGVzIHRvIGRyYXcKICAjIHJldHVybnMgbGlzdCB3aXRoIGVhY2ggZWxlbWVudCBhIHN1YnNldCBvZiBkYXRhc2V0CiAgCiAgYm9vdF9hY2N1bSA8LSBsaXN0KCkKICBmb3IoaSBpbiAxOm5fcmVzYW1wbGVzKXsKICAgIGJvb3RfYWNjdW1bW2ldXSA8LSBkYXRhc2V0IHw+CiAgICAgIHNsaWNlX3NhbXBsZShuID0gcmVzYW1wbGVfc2l6ZSwgcmVwbGFjZSA9IFRSVUUpCiAgfQogIHJldHVybihib290X2FjY3VtKQp9CgoKcGxvdF9ib290X3NhbXBsZSA8LSBmdW5jdGlvbihib290X3NhbXBsZSl7CiAgIyBib290X3NhbXBsZTogZGF0YXNldCwgb3V0Y29tZSBvZiBkcmF3X2Jvb3Rfc2FtcGxlCiAgYm9vdF9zYW1wbGUgfD4KICAgIGdncGxvdChhZXMoeCA9IHgsIHkgPSB5X3Zpb2wpKSArCiAgICBnZW9tX3BvaW50KCkgKwogICAgZ2VvbV9zbW9vdGgobWV0aG9kID0gJ2xtJywgc2UgPSBGQUxTRSkgKwogICAgeGxpbSgtMiwgMikgKwogICAgTlVMTAogIAp9CgpzZXQuc2VlZCgxKQpib290X3NhbXBsZXMgPC0gZHJhd19ib290X3NhbXBsZXMob3JpZ19kYXRhLCBib290X3NhbXBsZV9zaXplLCBuX3Jlc2FtcGxlcykKYm9vdF9wbG90X2xpc3QgPC0gbGFwcGx5KGJvb3Rfc2FtcGxlcywgcGxvdF9ib290X3NhbXBsZSkKCiMgcGxvdF9ib290X3NhbXBsZShib290X3NhbXBsZXNbWzFdXSkKd3JhcF9wbG90cyhib290X3Bsb3RfbGlzdFsxOjEyXSkKYGBgCgoKYGBge3IgbWVzc2FnZT1GfQpmaXRfbG0gPC0gZnVuY3Rpb24oYm9vdF9zYW1wbGUpewogICMgYm9vdF9zYW1wbGU6IGRmIHdpdGggY29scyB4LCB5X3Zpb2wKICAjIHJldHVybnMgY29lZnMgb2YgTE0gZml0IHRvIGJvb3Rfc2FtcGxlCiAgCiAgbSA8LSBsbSh5X3Zpb2wgfiB4LCBkYXRhID0gYm9vdF9zYW1wbGUpCiAgcmV0dXJuKG0kY29lZmZpY2llbnRzKQp9CgojIGZpdF9sbShib290X3NhbXBsZXNbWzFdXSkKYm9vdF9jb2VmcyA8LSBsYXBwbHkoYm9vdF9zYW1wbGVzLCBmaXRfbG0pCmJvb3RfY29lZnNfZGYgPC0gYm9vdF9jb2VmcyB8PgogIGJpbmRfcm93cyguaWQgPSAnYm9vdF9pZHgnKQoKYm9vdF9pbnRfbWVhbiA8LSBtZWFuKGJvb3RfY29lZnNfZGYkYChJbnRlcmNlcHQpYCkKYm9vdF9pbnRfc2QgPC0gc2QoYm9vdF9jb2Vmc19kZiRgKEludGVyY2VwdClgKQpib290X3NscF9tZWFuIDwtIG1lYW4oYm9vdF9jb2Vmc19kZiR4KQpib290X3NscF9zZCA8LSBzZChib290X2NvZWZzX2RmJHgpCgoKcF9pbnRlcmNlcHQgPC0gYm9vdF9jb2Vmc19kZiB8PgogIGdncGxvdChhZXMoeCA9IGAoSW50ZXJjZXB0KWApKSArCiAgZ2VvbV9oaXN0b2dyYW0oZmlsbCA9ICdncmV5JykgKwogIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IGJvb3RfaW50X21lYW4sIGxpbmV3aWR0aCA9IDIsIGNvbG91ciA9ICdibGFjaycpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X2ludF9tZWFuICsgYm9vdF9pbnRfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2Rhc2hlZCcpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X2ludF9tZWFuIC0gYm9vdF9pbnRfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2Rhc2hlZCcpICsgCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gYm9vdF9pbnRfbWVhbiArIDEuOTYqYm9vdF9pbnRfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2RvdHRlZCcpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X2ludF9tZWFuIC0gMS45Nipib290X2ludF9zZCwgbGluZXdpZHRoID0gMSwgY29sb3VyID0gJ2JsYWNrJywgbGluZXR5cGUgPSAnZG90dGVkJykgKwogIGdlb21fZnVuY3Rpb24oZnVuID0gZnVuY3Rpb24oeCkgZG5vcm0oeCwgbWVhbiA9IGJvb3RfaW50X21lYW4sIHNkID0gYm9vdF9pbnRfc2QpICogMywgY29sb3VyID0gJ2JsYWNrJykgKwogIE5VTEwKCnBfc2xvcGUgPC0gYm9vdF9jb2Vmc19kZiB8PgogIGdncGxvdChhZXMoeCA9IHgpKSArCiAgZ2VvbV9oaXN0b2dyYW0oZmlsbCA9ICdncmV5JykgKwogIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IGJvb3Rfc2xwX21lYW4sIGxpbmV3aWR0aCA9IDIsIGNvbG91ciA9ICdibGFjaycpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X3NscF9tZWFuICsgYm9vdF9zbHBfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2Rhc2hlZCcpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X3NscF9tZWFuIC0gYm9vdF9zbHBfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2Rhc2hlZCcpICsgCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gYm9vdF9zbHBfbWVhbiArIDEuOTYqYm9vdF9zbHBfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2RvdHRlZCcpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X3NscF9tZWFuIC0gMS45Nipib290X3NscF9zZCwgbGluZXdpZHRoID0gMSwgY29sb3VyID0gJ2JsYWNrJywgbGluZXR5cGUgPSAnZG90dGVkJykgKwogIGdlb21fZnVuY3Rpb24oZnVuID0gZnVuY3Rpb24oeCkgZG5vcm0oeCwgbWVhbiA9IGJvb3Rfc2xwX21lYW4sIHNkID0gYm9vdF9zbHBfc2QpICogMywgY29sb3VyID0gJ2JsYWNrJykgKwogIE5VTEwKCnBfaW50ZXJjZXB0ICsgcF9zbG9wZQpgYGAKCgpeIHdvdWxkIGJlIGNvb2wgdG8gYW5pbWF0ZSB0aGUgd2F5IHRoZXNlIHNhbXBsaW5nIGRpc3RyaWJ1dGlvbnMgY2hhbmdlIGFzIHNhbXBsZXMgdHJpY2tsZSBpbiwgdGhlIHdheSBJIGRpZCBmb3IgYmF5ZXNfc3RhdC4KCgpUaGUgYmlnIGRpZmZpY3VsdHkgd2l0aCBib290c3RyYXBwaW5nIGlzIHRoYXQgdG8gYWN0dWFsbHkgdW5kZXJzdGFuZCBXSFkgaXQgd29ya3MsIHlvdSBuZWVkIHRvIHVuZGVyc3RhbmQgdGhlIHZlcnkgdmVyeSBjb21wbGljYXRlZCBtYXRocyBiZWhpbmQgaXQuClNvIGl0J3Mgbm90IGVhc3kgdG8gZ2l2ZSBhIGdvb2QgaW50dWl0aW9uIGZvciB3aHkgaXQgaGVscHMuCgoK